const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  let totalBlogs = [];

  const tab = 'other';
  const sub_tab = 'hollis';
  const url = 'https://www.hollischuang.com/list';
  const fileName = `hollis.json`;


  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  ;


  await page.goto(url);
  await page.waitFor('body > div.container.container-page > div.content > article.archives > div > ul > li > a');


  totalBlogs = await page.$$eval('body > div.container.container-page > div.content > article.archives > div > ul > li > a', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.innerText
      let content = ele.href
      tmpBlogs.push({
        title: title,
        content: content,
        tab: 'other',
        sub_tab: 'hollis'
      });
    });
    return tmpBlogs;
  });
  console.log(totalBlogs);


  let writerStream = fs.createWriteStream(`crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');


};


main();


